#
# For each shrinkage regime, prints:
# 1. most and least heavily weighted similarity indicators
# 2. similarity weights for a number of hand-picked example terms.
# 3. similarity weight stats for some hand-picked movies.
#

require 'cgi'

#
# Common data and methods.
#

# Will need to look up titles for terms; load the big.titles list.
unless $title
  $title = File.new('big.titles').readlines
end

# Will need to look up which movies terms occurred on.
# Note that the values of term_movies are Netflix movie ids, not Wikipedia
# pages. The term_page_counts values are the number of matched Wikipedia pages.
# To complicate matters, we have several versions of this: one for the "all
# matches" matching one for the "one to n" matchings.
# The other matchings are one_to_2.new one_to_3.new -- dropped to save space.
$matchings = %w(new one_to_1.new)
if $term_movies == nil || $term_page_counts == nil
  $term_movies = {}
  $term_page_counts = {}
  $matchings.each do |matching|
    links_by_movie = File.new("big.#{matching}.links_by_movie").readlines.
      map{|s|
        term, num_pages, num_movies, *movies = s.split(' ')
        [term.to_i, num_pages.to_i, movies.map{|i| i.to_i}]
      }
    $term_movies[matching] = Hash[
      links_by_movie.map{|term,num_pages,movies| [term,movies]}]
    $term_page_counts[matching] = Hash[
      links_by_movie.map{|term,num_pages| [term,num_pages]}]
  end
end

# Will also be useful to look at the reverse map, from movies to terms.
# Again, we have two of these: one for "new" and one for "one to one".
unless $movie_terms
  $movie_terms = {}
  $matchings.each do |matching|
    $movie_terms[matching] = Hash.new {|h,k| h[k] = []}
    $term_movies[matching].each do |term, movies|
      movies.each do |movie|
        $movie_terms[matching][movie] << term
      end
    end
  end
end
# Will need to map netflix movie ids to Wikipedia pages.
unless $movie_pages
  $movie_pages = Hash[File.new('big.new.movies.with_ids').readlines.map{|s|
    s.split(' ').map{|i| i.to_i}
  }]
end
# It's also worth looking up the Netflix titles, while we're at it.
unless $movie_titles
  $movie_titles = Hash[File.new('data/movie_titles.txt').readlines.map{|s|
    m,year,title = s.split(',')
    [m.to_i, title.chomp]
  }]
end

# Some simple extensions.
module Enumerable
  def sum
    inject(0) {|s,v| s+v}
  end
  def mean
    sum / (size + 0.0)
  end
  def median
    self.sort[size / 2]
  end
  def variance
    avg=self.mean
    sum=self.inject(0){|acc,i|acc +(i-avg)**2}
    return(sum/self.size.to_f)
  end
end

#
# JLM HACK 20081124
# The department machine has some security problems, so we have to disable a
# warning. This is based on
# http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/223512
#
def quietly
  v = $VERBOSE
  $VERBOSE = nil
  yield
ensure
  $VERBOSE = v
end

# Invoke make and swallow the output.
def make s
  quietly do
    `make #{s}`
  end
end

#
# Find term_weights results files and parse their names to recover the
# parameters in a more easily digestible form.
#
def load_term_weights filter=nil
  # Find term weights results files.
  tw_files = Dir['*.term_weights']

  # Filter based on only those passed as arguments, if any filter given.
  if filter && filter.size > 0 
    tw_files = tw_files.select {|f| filter.member?(f)}
  end

  # Parse file names to recover parameters [matching, beta, d] (see
  # term_weights.cpp for meanings).
  tws = tw_files.map {|f|
    matching = "new"
    if f =~ /^one_to_(\d+)_(.+)$/
      matching = "one_to_#{$1}.new"
      f = $2
    end
    if f =~ /^d(\d+)_beta(\d+).term_weights$/
      [matching,$2.to_i,$1.to_i]
    else
      raise "failed to parse #{f}"
    end
  }

  return [tw_files, tws]
end

# Utility method for processing term lists, below.
def term_lines_lookup lines, all_exist=true
  # Need reasonably efficient access to inverse titles map.
  unless $title_inv
    $title_inv = {}
    (0...$title.size).each do |i|
      $title_inv[$title[i]] = i
    end
  end

  lines = lines.split("\n").map{|s|s.strip}.reject{|s| s.size==0}
  return lines.map {|s|
    number = $title_inv["#{s}\n"]
    raise "term #{s} not found" unless number
    raise "term #{s} not used; is it a redirect?" unless
      $term_movies['new'][number] || !all_exist
    number
  }
end

# Look up these example terms by title.
# Many of these come from the 72nd Academy Awards (winners and nominees).
# The list of Star Trek TNG actors was taken from the TNG wiki page.
# I also added Colm Meany and Majel Barrett.
# And Brad Pitt, for comparison with the paper.
#$example_actors = term_lines_lookup('
#Kevin Spacey
#Hilary Swank
#Michael Caine
#Angelina Jolie
#Russell Crowe
#Sean Penn
#Denzel Washington
#Richard Farnsworth
#Annette Bening
#Julianne Moore
#Meryl Streep
#Janet McTeer
#Michael Clarke Duncan
#Haley Joel Osment
#Tom Cruise
#Jude Law
##Toni Collette
#Catherine Keener
#Chloë Sevigny
#Samantha Morton
#Brad Pitt
#Patrick Stewart
#Jonathan Frakes
#Brent Spiner
#LeVar Burton
#Marina Sirtis
#Michael Dorn
#Gates McFadden
#Denise Crosby
#Wil Wheaton
#Colm Meaney
#Majel Barrett
#') unless $example_actors

# Look up actors in file - there are quite a few of them.
$example_actors = term_lines_lookup(
  File.open('actors.titles') {|f| f.read}, false) unless $example_actors

# These genres came from the list of genres on the "Film genre" page.
# I also added Documentary film and Romantic comedy, for my own interest.
$example_genres = term_lines_lookup('
Action film
Adventure film 
Animation
Biographical film
Comedy
Children\'s film
Crime film
Disaster film
Documentary film
Drama
Fantasy film
Horror film
Musical film
Romantic comedy
Science fiction
Short subject
Sport
Thriller (genre)
War film
Western (genre)
') unless $example_genres

$example_awards = term_lines_lookup('
Academy Award
Academy Award for Best Picture
Academy Award for Best Actor
Academy Award for Best Actress
Academy Award for Best Director
British Academy of Film and Television Arts
BAFTA Award for Best Film
BAFTA Award for Best Actor in a Leading Role
BAFTA Award for Best Actress in a Leading Role
BAFTA Award for Best Direction
Screen Actors Guild Awards
') unless $example_awards

# These are just some random words I thought up.
$example_terms = term_lines_lookup('
Star Trek
War
Peace
Sex
Love
Hatred
Jesus
Animal
Gun
President
Government
Racism
Boat
Fixed-wing aircraft
Train
Automobile
Darkness
Light
Hero
Cowardice
') unless $example_terms

# Running out of memory! Drop the inverse titles map.
$title_inv = nil

# Look at what kinds of words occur on a few "interesting" movies that I've
# chosen.
def movie_titles_to_ids titles
  titles = titles.split("\n").map{|t| t.strip}.reject{|t| t.size() == 0}
  lookup = $movie_titles.invert
  titles.map {|t|
    id = lookup[t]
    raise "movie #{t} not found" unless id
    id
  }
end

# Most of these are from the discussion at
# http://www.netflixprize.com/community/viewtopic.php?id=164
$example_movies = movie_titles_to_ids('
Miss Congeniality
Independence Day
The Patriot
The Day After Tomorrow
The Stepford Wives
Gigli
Full Frontal
Solaris
Birth
Sky Captain and the World of Tomorrow
Pirates of the Caribbean: The Curse of the Black Pearl
The Shawshank Redemption: Special Edition
Lord of the Rings: The Return of the King
The Green Mile
Finding Nemo (Full-screen)
Finding Nemo (Widescreen)
Raiders of the Lost Ark
Forrest Gump
The Royal Tenenbaums
Lost in Translation
Pearl Harbor
Napoleon Dynamite
Fahrenheit 9/11
')

# Find titles for highest- and lowest-weighted n terms in term_weights file
# twf. Also prints the example terms, because it's pretty much the same code.
# Prints this a tables, each in its own td (a hack).
def print_top_bot_examples n, matching, twc
  twc = twc.sort_by{|t,w| -w}.
    zip((1..twc.size).to_a).
    map{|twcs,rank| twcs+[rank]}
  tw_top = twc[0...n]
  tw_bot = twc.reverse[0...n]
  tw_ex = [$example_actors,$example_genres,
    $example_awards,$example_terms].map{|twex|
    twc.select{|t,w| twex.member?(t)}}
  ([tw_top,tw_bot]+tw_ex).each do |twc|
    puts "<td valign=\"top\"><table>"
    puts "<tr><th>"
    puts %w(rank w pages movies coratings term).join('</th><th>')
    puts "</th></tr>"
    twc.each do |t,w,u_sum,rank|
      puts "<tr><td align=\"right\">%d</td>" % rank
      puts "<td align=\"right\">%.3f</td>" % w
      puts "<td align=\"right\">%d</td>" % $term_page_counts[matching][t]
      puts "<td align=\"right\">%d</td>" % $term_movies[matching][t].size
      puts "<td align=\"right\">%d</td>" % u_sum.to_i
      title = CGI.escapeHTML($title[t].chomp)
      back_links = $term_movies[matching][t].map{|m|
        "<tr><td nowrap>#{CGI.escapeHTML($title[$movie_pages[m]])}</td>" +
            "<td nowrap>#{CGI.escapeHTML($movie_titles[m])}</td></tr>"}
      puts <<TITLE;
<td nowrap>
  <a href="http://en.wikipedia.org/wiki/#{title}" class="info">#{title}
  <span>
    <table>
      <tr><th>page</th><th>movie</th></tr>
      #{back_links.join("\n")}
      </table></span></a>
</td></tr>
TITLE
    end
    puts "</table></td>"
  end
  nil
end

# Look at stats on words appearing on particular movies
def print_movie_stats matching, twc
  twh = Hash[twc.map{|t,w|[t,w]}]
  data = $example_movies.map{|m|
    terms = $movie_terms[matching][m]
    weights = terms.map{|t| twh[t]}.compact
    if terms.empty?
      [m,terms.size, weights.size, 0, 0]
    else
      [m,terms.size, weights.size, weights.mean, weights.variance]
    end
  }
  data = data.sort_by {|m,ts,ws,wm,wv| -wm}

  puts "<td valign=\"top\"><table>"
  puts "<tr>"
  puts "<th>" + %w(movie terms weights mean stdev).join("</th><th>") + "</th>"
  puts "</tr>"
  data.each do |m,ts,ws,wm,wv|
    puts "<tr>"
    puts "<td nowrap>#{$movie_titles[m][0...20]}</td>"
    puts "<td align=\"right\">#{ts}</td>"
    puts "<td align=\"right\">#{ws}</td>"
    puts "<td align=\"right\">%.3f</td>" % wm
    puts "<td align=\"right\">%.3f</td>" % Math::sqrt(wv)
    puts "</tr>"
  end
  puts "</table></td>"
  nil
end

# Write report for all files.
def report
  # Filter based on only those passed as arguments, if any args given.
  tw_files, tws = load_term_weights(ARGV)

  tws.map{|matching,params| matching}.all? {|m| $matchings.member?(m)} or
    raise "not all matchings are recognized"

  puts <<HEADER; 
  <html><head><title>term_weights</title>
  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
  <!-- CSS tooltip stolen from http://psacake.com/web/jl.asp -->
  <style>
  a.info{
    position:relative; /*this is the key*/
    z-index:24;
    color:#000;}

      a.info:hover{z-index:25; background-color:#fff}

        a.info span{display: none}

    a.info:hover span{ /*the span will display just on :hover state*/
      display:block;
    position:absolute;
    top:2em; left:2em;
    border:1px solid #000;
    background-color:#fff;
      padding:5px;
    }
  </style>
  </head><body>
  <table><tr>
HEADER
  headers = %w(
matching
beta
d
density
top_50
bot_50
actors
genres
awards
misc
movies)
  puts "<th>"+headers.join('</th><th>')+"</th>"
  puts "</tr>"
  tws.zip(tw_files).sort.each do |tw,twf|
    matching = tw[0]

    puts "<tr><td>" + tw.join('</td><td>') + "</td>"
    make "rep/#{twf}.density.png"
    puts "<td>"
    puts "<img src=\"#{twf}.density.png\"/>"
    # Some basic stats are also useful.
    twc = File.new(twf).readlines.map{|s|s.split(' ')}.map{
      |t,w,u_sum|[t.to_i,w.to_f,u_sum.to_f]}
    terms = twc.map{|t,w,u_sum|t}
    weights = twc.map{|t,w,u_sum|w}
    page_counts = terms.map{|t| $term_page_counts[matching][t]}
    movie_counts = terms.map{|t| $term_movies[matching][t].size}
    coratings = twc.map{|t,w,u_sum|u_sum.to_i}

    puts "mean: %.3f<br/>" % weights.mean
    puts "median: %.3f<br/>" % weights.median
    puts "mean pages: %.3f<br/>" % page_counts.mean
    puts "median pages: %d<br/>" % page_counts.median
    puts "mean movies: %.3f<br/>" % movie_counts.mean
    puts "median movies: %d<br/>" % movie_counts.median
    puts "mean coratings: %.3f<br/>" % coratings.mean
    puts "median coratings: %d<br/>" % coratings.median
    puts "</td>"
    print_top_bot_examples 50, matching, twc
    print_movie_stats matching, twc
    puts "</tr>"
  end
  puts "</table></body></html>"
end

if __FILE__ == $0
  report
end

# Copyright (c) 2009 John Lees-Miller
# 
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
# 
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
# 
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.

